<?php
class Crawl_sina_keyword_from_html_library  extends Crawl_keyword_library{

    private $_header;
    private $_userAgent;
    private $_url;
    private $_cookie;
    private $_options;
    private $_htmlsearch_config;
    private $_crawl_array;
    private $_parameters;
    const ONLY_CRAWL_HOT_PAGE_CRITICAL_POINT = 1;

    /**
     * 定义 CURL 消息头、用户端信息、抓取地址
     * 定义 新浪网页抓取选项
     */
    function __construct(){
        parent::__construct();

        $openplatforms = $this->_ci->config->item('openplatforms');
        $this->_htmlsearch_config  = $openplatforms['openplatforms']['htmlsearch']['sina_html'];
        $this->_header      = $this->_htmlsearch_config['header'];
        $this->_userAgent   = $this->_htmlsearch_config['user_agent'];
        $this->_referer   = $this->_htmlsearch_config['referer'];
        $this->_url         = $this->_htmlsearch_config['url'];
        $this->_illegal_tag  = $this->_htmlsearch_config['illegal_tag'];
        $this->_result_error = $this->_htmlsearch_config['result_error'];
        $this->_result_success = $this->_htmlsearch_config['result_success'];
        $this->_options     = $this->_htmlsearch_config['xsort'];
        $this->_options_mini = $this->_htmlsearch_config['mini_xsort'];
        $this->_null_tab    = $this->_htmlsearch_config['null'];
        $this->write_log_stary_tag();
    }


    public function crawl($retry = FALSE) {

        if($this->_ci->input->post('page') !== FALSE && $this->_ci->input->post('page') > self::ONLY_CRAWL_HOT_PAGE_CRITICAL_POINT) {
            $this->_options = $this->_options_mini;
        }

        $this->_crawl_array = array();
        $null_result['data'] = NULL;
        $crawl_result = array();
        $error_array = array();
        foreach($this->_options as $level => $order) {
            $result = $this->try_crawl($retry, $order);
            if(is_array($result) && array_key_exists('data', $result) && is_null($result['data'])) {
                $null_result[$level] = $result;
            } else if(is_array($result) && array_key_exists($this->_error_string, $result)){
                $error_array[] = $result;
            } else {
                $crawl_result[$level] = $result;
            }
        }

        if(count($crawl_result) > 0) {
            return $this->make_data($crawl_result);
        } else if(count($error_array) > 0) {
            return $error_array[count($error_array) - 1];
        } else {
            return $null_result;
        }
    }

    /**
     * 组织数据
     * @param array $result_array
     * @return array
     */
    private function make_data($result_array = array()) {
        $data_array = array();
        $error_string = '';
        $this->_ci->load->model('split/htmlsearchparser_model');
        $this->_crawl_tab_array = array();
        foreach($result_array as $crawl_use_tab => $data) {
            if(is_array($data) && array_key_exists($this->_error_string, $data)) {
                $error_string .= $crawl_use_tab.'--'.$data[$this->_error_string].' ';
            } else {
                $split_data = $this->_ci->htmlsearchparser_model->parser_sina_html($this->_htmlsearch_config, $data);
                $this->_ci->htmlsearchparser_model->clear();
                if($split_data != FALSE) {
                    $data_array = $this->merge_data($split_data, $crawl_use_tab);
                } else {
                    $error = $this->_ci->error_code_library->getError('system', 'common', 'match_split_data');
                    log_message(self::LOG_LEVEL_WARING, 'Crawl From HTML Result And Split Data Failed: '.json_encode($error), FALSE, $this->_log_dir);
                    $error_string .= ' '.$crawl_use_tab.'='.json_encode($error);
                }
            }
        }

        if(count($this->_crawl_array) < 1) {
            $error = $this->_ci->error_code_library->getError('system', 'common', 'merge_data', $error_string);
            log_message(self::LOG_LEVEL_WARING, 'Crawl From HTML Result Merge Data Failed: '.json_encode($error), FALSE, $this->_log_dir);
            return $error;
        } else {
            return array_values($this->_crawl_array);
        }

    }

    private function merge_data ($split_data, $crawl_use_tab) {
        foreach($split_data as $key => $data) {
            if(array_key_exists($key, $this->_crawl_array) === FALSE) {
                $data['source_myid'] = $crawl_use_tab;
                $this->_crawl_array[$key] = $data;
            }
        }
        return $this->_crawl_array;
    }


    /**
     * 尝试抓取
     * @param bool $retry 是否重试
     * @param $order      抓取页面选项卡
     * @return array|bool 抓取成功返回数组
     */
    private function try_crawl($retry = FALSE, $order) {
        $parameters = $this->make_parameters($order);

        if(is_array($parameters) && array_key_exists($this->_error_string, $parameters)) {
            log_message($this->_error_string, 'Crawl From Sina Html Make CURL Parameters Error', FALSE, $this->_log_dir);
            return $parameters;
        }
        if($retry == FALSE) {
            //不使用代理、不重试直接抓取
            log_message(self::LOG_LEVEL_NOTICE, 'Will Crawl From Sina Html Un Use Retry And Un Use Proxy With Parameters: '.json_encode($parameters), FALSE, $this->_log_dir);
            return self::get_content($parameters);
        } else {
//             $crawl_use_proxy_ip_result = $this->crawl_use_proxy_ip($this->_parameters, self::PRROXY_IP, $this->_crawl_from['sina_html'], FALSE);
//             if(is_array($crawl_use_proxy_ip_result) && array_key_exists($this->_error_string, $crawl_use_proxy_ip_result) && $this->not_illegal_error($crawl_use_proxy_ip_result)) {
                return $this->crawl_use_proxy_ip($this->_parameters, self::MOBILITY_IP, $this->_crawl_from['sina_html'], FALSE);
//             } else {
//                 return $crawl_use_proxy_ip_result;
//             }
        }
    }

    /**
     * 正则验证抓取结果 重写父类方法
     * 判断标标准20130524 如果能匹配到 验证码输入况 或者匹配不到HTML结束标签 认为 失败 否则 成功
     * @param $html  抓取到的数据
     * @return bool  失败 FALSE 成功 TRUE
     */
    public function match_html($html) {
        if(strpos($html, $this->_result_error)) {
        	//如果返回登陆页 或 被禁 标记cookie 为不可用
        	
        	//如果使用了cookie，标记不可用，并重新组织参数 更新COOKIE
        	if(!empty($this->_cookie)){
        		$this->label_cookie($this->_cookie['id']);
        		
        		$this->make_parameters();
        	}
        	
            $this->label_proxy_table($this->_proxy_ip, $this->_use_proxy);
            
            return $this->_ci->error_code_library->getError('curl', 'data', 'verification_code', $this->_result_error);
        } elseif(preg_match($this->_illegal_tag, $html)) {;
            //关键词被对应平台判断为非法
            return $this->_ci->error_code_library->getError('curl', 'data', 'illegal', 'From '.$this->_crawl_from['sina_html'].$this->_illegal_tag);
        } elseif(strpos($html, $this->_result_success) === FALSE) {
            return $this->_ci->error_code_library->getError('curl', 'data', 'incomplete_answers', $this->_result_success);
        } else {
            return TRUE;
        }
    }

    /**
     * 拼接抓取url
     * @param string $order 新浪抓取选项卡参数 array('p1' => NULL , 'p2' => 'time', 'p3' => 'hot');
     * @return array 包含URL的参数数组
     */
    private function make_parameters($order = '') {
        $parameters['url'] = $this->_url.urlencode(trim($this->_ci->input->post('word')));

        //$parameters['httpHeader'] = $this->_header;
        if($this->_referer) $parameters['referer'] = $this->_referer;
        if($this->_htmlsearch_config['time_out']) $parameters['time_out'] = $this->_htmlsearch_config['time_out'];

        $page = $this->get_page();

        if(!empty($order)) {
            $parameters['url'] = $parameters['url'].'&xsort='.$order.'&page='.$page.'&nodup=1';
        } else {
            $parameters['url'] = $parameters['url'].'&page='.$page.'&nodup=1';
        }

		//时间范围条件
		$timescope = trim($this->_ci->input->post('timescope'));
		if($timescope){
			$parameters['url'] .= '&timescope='.$timescope;
		}
		
		//由于新浪改版，调整为 page > 1 时，使用cookie抓取
		/*if( $page > 1){
			$cookie = $this->get_cookie();
			
			if(is_array($cookie) && array_key_exists($this->_error_string, $cookie)) {
				return $cookie;
			}
			
			$this->_cookie = $cookie;
			
			$parameters['cookie_str'] = true;
			$parameters['cookie'] = $this->_cookie['cookie_str'];
			
			$this->update_cookie_use_count($cookie['id']);
		}*/

        $this->_parameters = $parameters;
        return $this->_parameters;
    }

    /**
    * unlock_ip
    *
    * @access public
    * @return string $mixed
    */
    public function unlock_ip() {
        $php_path = '/usr/local/php/bin/php';

        $file = '/home/www/crawl/public/index.php unlock_ip multi_test ff';
        //$file = "/home/www/crawl/public/index.php unlock_ip unlock ".json_encode($this->_proxy_ip['use_ip']);

        $command = $php_path.' '.$file.' >> /root/unlock1.log &';
        echo $command;
        exec($command,$output,$return);
    }
}
